package com.gxljc.bear.crawler.itaogao.cscomcn;

import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.base.DolphinFetchData;
import com.gxljc.bear.crawler.itaogao.ItaogaoConst;
import com.gxljc.bear.crawler.itaogao.NewsPageTable;
import com.gxljc.bear.crawler.itaogao.util.MongodbUtil;
import com.gxljc.bear.crawler.proxy.ProxyUtil;
import com.gxljc.bear.crawler.util.DateUtil;
import com.gxljc.bear.crawler.util.HtmlUtil;
import com.gxljc.bear.crawler.util.MD5Util;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.Serializable;
import java.util.*;

/**
 * 中国证券报 seed 数据爬虫。
 *
 * @author tanghaitao
 * @since 2022-9-14
 */
public class CscomcnSeedCrawler implements Serializable {
    public static Logger LOG = Logger.getLogger(CscomcnSeedCrawler.class);
    private Boolean proxy = false;
    final int timeOut = 30000;

    public CscomcnSeedCrawler(Boolean proxy) {
        this.proxy = proxy;
    }

    public void crawl() throws Exception {
        List<String> seeds = getSeed();
        if (CollectionUtils.isEmpty(seeds)) {
            LOG.error("page is empty");
            return;
        }
        crawlNoSpark(seeds);
    }

    public void crawlNoSpark(List<String> seeds) {
        ProxyUtil.initQueue(proxy);
        for (String url : seeds) {
            try {
                crawl(url);
//                break;
            } catch (Exception e) {
                e.printStackTrace();  //To change body of catch statement use File | Settings | File Templates.
            }
        }
    }

    //单条url入口
    public int crawl(String url) throws Exception {
        LOG.info("crawl url = " + url);
        DolphinFetchData fetchData = DolphinFetchData.getInstance(DolphinCrawlerConsts.CrawlerChannelType.bear.getName());
        byte[] htmlByte = fetchData.getHtml(url, proxy, timeOut);
        if(htmlByte == null){
            LOG.error("page result is empty url = " + url);
            return DolphinCrawlerConsts.RESULT_NO;
        }
        String html = new String(htmlByte,"utf-8");
        List<NewsPageTable> tables = extract(html, url);
        MongodbUtil.saveMongodb(tables, ItaogaoConst.MONGODB_TABLE.CSCOMCN.getValue());
        return DolphinCrawlerConsts.RESULT_YES;
    }

    private List<NewsPageTable> extract(String html, String url) throws Exception {
        Document doc = Jsoup.parse(html);
        Elements tables = doc.select("td div table");
        if(CollectionUtils.isEmpty(tables)) return null;
        List<NewsPageTable> all = new LinkedList<>();
        for(Element table : tables){
            Elements font = table.select("td font");
            if(CollectionUtils.isEmpty(font)) continue;
            String txt = font.text();
            if(txt.contains("信息披露")) continue;
            List<NewsPageTable> list = parseObject(table, url);
            if(CollectionUtils.isEmpty(list)) continue;
            all.addAll(list);
        }
        return all;
    }

    private List<NewsPageTable> parseObject(Element aelement, String url) throws Exception {
        Elements aList = aelement.select("a");
        if(CollectionUtils.isEmpty(aList)) return null;
        List<NewsPageTable> tables = new LinkedList<>();
        for(Element a : aList){
            String crawlUrl = a.attr("href");
            crawlUrl = HtmlUtil.pictureUrlPref(crawlUrl, url);
            NewsPageTable table = new NewsPageTable();
            table.setId(genId(crawlUrl));
            table.setCrawlUrl(crawlUrl);
            tables.add(table);
        }
        return tables;
    }

    //生成id
    private String genId(String href) {
        return MD5Util.getMd5(href);
    }

    //构造URL seed
    public List<String> getSeed() throws Exception {
        List<Date> dateSeed = getDateSeed();
        if (CollectionUtils.isEmpty(dateSeed)) return null;
        List<String> urls = new LinkedList<String>();
        for (Date date : dateSeed) {
            String dateFormat = DateUtil.dateFormat(date, "yyyy-MM/dd");
            //http://epaper.cs.com.cn/html/2016-09/14/node_17.htm
            String url = String.format(ItaogaoConst.NEWSPAPER_SEED.CSCOMCN.getValue(), dateFormat);
            urls.add(url);
        }
        return urls;
    }

    //获取从当前到1月1号的列表
    private List<Date> getDateSeed() {
        Calendar now = Calendar.getInstance();
        List<Date> list = new LinkedList<>();
        while (true) {
            int month = now.get(Calendar.MONTH);
            int day = now.get(Calendar.DAY_OF_MONTH);
            int week = now.get(Calendar.DAY_OF_WEEK);
            list.add(now.getTime());
            if (month == 0 && day == 1) {
                return list;
            }
            now.add(Calendar.DATE, -1);
        }
    }
}
